// var request = require('superagent')
var request = require('request')
var iconv = require('iconv-lite')
var fs = require('fs')
var path = require('path')

var url = "http://www.dybird.com/down/xiazai66179.html"
// var encoding = 'gb2312'
var encoding = "utf-8"

// var rg = new RegExp(/ftp\S*.rmvb/ig)
var rg = new RegExp(/http\S*.mp4/ig)

var urls = []
request.get(url, { gzip: true, encoding: null }, function (error, response, body) {
    if (error) {
        console.error(error)
    } else {
        // console.log(body)
        var lines = iconv.decode(body, encoding).split("\n")
        lines.forEach(line => {
            if(rg.test(line)) {
                var tmpUrls = line.match(rg)
                tmpUrls.forEach(tmpUrl => {
                    urls.push(tmpUrl)
                })
            }
        })
        fs.writeFileSync(path.join(__dirname, 'urls.txt'),
            urls.join("\n") + "\n"
        )
    }
})

